package util.ranker;

import java.io.IOException;
import java.security.Key;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map.Entry;

import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.queryParser.ParseException;

import util.hashing.Sorting;
import util.io.FileInput;
import util.ir.LMSearcherMix;
import util.models.LanguageModel;
import util.models.Tags;

public class TagRankingMix {

	private static final int TOP_N = 200;

	public static String index_raw = "/home/sergio/projects/delicious_index/bigrams_mix_clean2/bigrams_mix_clean2.txt";
	public static String bigram_index = "/home/sergio/delicious_index/bigrams_index_mix_clean2";
	public static String trigram_index = "/home/sergio/delicious_index/trigrams_index_mix";

	float lambda = 0.99f;

	/**
	 * Receives tags representing the query The seed tags
	 * 
	 * Output
	 * 
	 */

	static Tags tags = new Tags();

	Hashtable<String, Float> seed_tags = new Hashtable<String, Float>();
	LMSearcherMix trigram_searcher = null;
	LMSearcherMix bigram_searcher = null;

	Hashtable<String, Float> query_hash = null;
	HashSet<String> t_tags = new HashSet<String>();

	String type_foreground = "/";
	String type_foreground_b = "/";
	String type_background = "/";

	float query_total = 0;

	private LanguageModel lm_foreground = new LanguageModel();

	private LanguageModel lm_background = new LanguageModel();

	private LanguageModel lm_foreground_b = new LanguageModel();

	private LanguageModel lm_background_b = new LanguageModel();

	// flag modes

	boolean query_weight = false; // weight inside the aggreagater for each t
	boolean candidate_query_weight = false; // candidate query weight

	boolean KL_CANDIDATE_SCORE = true;
	boolean IDF_WEIGHT = false;
	// boolean prob_t2=true;
	boolean prob_weight =true; // probability of s in the foreground
									// collection
	String query_text = "";

	public TagRankingMix(LMSearcherMix trigram_searcher,
			LMSearcherMix bigram_searcher, String type_fore, String type_back,
			String type_fore_b, Hashtable<String, Float> query_tags,
			String query_text) throws IOException, ParseException {
		// trigram_searcher = new LMSearcher(trigram_index);

		this.trigram_searcher = trigram_searcher;
		this.bigram_searcher = bigram_searcher;

		this.query_text = query_text;

		type_foreground = type_fore;
		type_background = type_back;
		this.type_foreground_b = type_fore_b;

		this.query_hash = query_tags;
		initQueryTags(query_hash);
		initLanguageModel(query_tags);

		// initTagsFrequency(index_raw);

	}

	public static void initTagsFrequency(String path) {
		// adults 0 1 123 19138 14231 292 5186 34153

		FileInput in = new FileInput(path);
		String line = in.readString();

		while (line != null) {

			String t[] = line.split("\t");

			tags.addFreq(t[0], t[1], Integer.valueOf(t[4]));
			tags.addFreq(t[0], t[2], Integer.valueOf(t[5]));

			tags.addFreq("back", t[1], Integer.valueOf(t[7]));
			tags.addFreq("back", t[2], Integer.valueOf(t[8]));

			line = in.readString();

		}

	}

	public void initQueryTags(Hashtable<String, Float> query) {

		Enumeration<String> keys = query.keys();
		while (keys.hasMoreElements()) {
			String key = keys.nextElement();
			t_tags.add(key);
			query_total = query_total + query.get(key);

		}
	}

	public void init(Hashtable<String, Float> t_tags,
			Hashtable<String, Float> seed_tags, String model_foreground,
			String model_background) throws CorruptIndexException, IOException,
			ParseException {
		initLanguageModel(t_tags);

		this.query_hash = t_tags;

		if (seed_tags != null) {
			initSeedTags(seed_tags);
		}

	}

	public void initLanguageModel(Hashtable<String, Float> t_tags)
			throws CorruptIndexException, IOException, ParseException {

		Enumeration<String> keys = t_tags.keys();

		HashSet<String> query = new HashSet<String>();
		while (keys.hasMoreElements()) {

			String t = keys.nextElement();
			query.add(t);

		}

		// trigram_searcher.trigramSearchLM(query, lm_foreground, lm_background,
		// type_foreground);
		bigram_searcher.bigramSearchLM(query, lm_foreground, lm_background,
				type_foreground);

		if (!type_foreground_b.equals(type_foreground)) {

			bigram_searcher.bigramSearchLM(query, lm_foreground_b,
					lm_background_b, type_foreground_b);

			lm_foreground.addLanguageModel(lm_foreground_b);
			lm_background.unionLanguageModel(lm_background_b);

		}
		// lm_background.addLanguageModel(lm_background_b);

		// System.out.println("trigram size:" + lm_foreground.trigrams.size());

	}

	public void initSeedTags(Hashtable<String, Float> s_tags)
			throws CorruptIndexException, IOException, ParseException {

		Enumeration<String> keys = s_tags.keys();

		while (keys.hasMoreElements()) {

			String seed = keys.nextElement();

			float val = s_tags.get(seed);

			seed_tags.put(seed, val);

		}

	}

	public boolean tagIsInQuery(String tag, String query) {

		String q[] = query_text.split("\\s+");

		String t[] = tag.split("_");

		HashSet<String> tokens_found = new HashSet<String>();

		for (int i = 0; i < q.length; i++) { // each query token

			String query_token = q[i];
			for (int j = 0; j < t.length; j++) { // each tag token
				String tag_token = t[j];
				if (tag_token.equals(query_token)) {
					
					tokens_found.add(tag_token);
				}

			}

			if (tokens_found.size() == (t.length))
				return true;

		}

		return false;

	}

	public Iterator<Entry<String, Float>> rankSuggestions() {

		/*
		 * For each candidate tag S
		 */

		Hashtable<String, Float> scores = new Hashtable<String, Float>();
		Iterator<String> s_enum = lm_foreground.candidates.iterator();
		// System.out.println("Candidates size " +
		// lm_foreground.candidates.size());

		while (s_enum.hasNext()) {

			String s = s_enum.next();

			/*
			 * Skips tokens that are in the query
			 */

			if (tagIsInQuery(s, query_text)) {
				continue;

			}

			float score = aggregate_t(s);
			/*
			 * Group by k
			 */

			// wieght by probability of s , or KL of s
			float freq_s_foreground = lm_foreground.getUnigramFreq(s);
			float freq_s_background = lm_background.getUnigramFreq(s);

			float prob_s_fore = freq_s_foreground / TOTALS.KIDS_UNIGRAM;
			float prob_s_back = freq_s_background / TOTALS.ADULTS_UNIGRAM;

			float prob_s = 0f;

			if (prob_weight) {

				prob_s = (float) Math.log(prob_s_fore);

			}

			// get candidate_query_weight

			float candidate_q_w = 0.0f;

			if (candidate_query_weight && query_hash.containsKey(s)) {
				candidate_q_w = (float) (query_hash.get(s) / query_total);

				candidate_q_w = (float) Math.log(candidate_q_w);
			}

			// score = score * candidate_q_w * prob_s;

			float kl_score = kl_score(prob_s_fore, prob_s_back);

			if (!KL_CANDIDATE_SCORE) {

				kl_score = 1.0f;
			}

			//System.out.println(s + "\t" + prob_s_fore + "\t" + prob_s_back	+ "\t" + kl_score);

			score = score + candidate_q_w + prob_s;
			score = score * kl_score;
			

			scores.put(s, score);
			// System.out.println(s + "\t"+ score);

		}

		// System.out.println("sorting..");

		Iterator<Entry<String, Float>> tuples = Sorting.sortHashNumericValues(
				scores, false);

		// System.out.println("tuple s " + tuples);

		// scores.clear();
		// scores = null;

		return tuples;
	}

	private float aggregate_t(String s) {
		// TODO Auto-generated method stub

		float total = 0f;

		Iterator<String> iter = t_tags.iterator();
		// for each term in the query

		while (iter.hasNext()) {

			/**
			 * 
			 * For each t in k given s
			 * 
			 */
			String t = iter.next();
			// System.out.println("s:" + s + "\tt:" + t);

			// Score for t tag

			// we are going to ignore t s that does not exists in the foreground
			// model
			// except the case in whith s is in the query set"
			if (lm_foreground.unigrams.containsKey(t) || (t.equals(s))) {
				float score = score_tag_mix(s, t, lambda);

				total = total + score;

			}

		}

		return total;

	}

	private float score_tag_mix(String s, String t, float lambda) {
		// TODO Auto-generated method stub

		float freq_st_foreground = lm_foreground.getBigramFreq(s, t); // check
		// order

		float freq_st_background = lm_background.getBigramFreq(s, t); // check
		// order

		float freq_t_foreground = lm_foreground.getUnigramFreq(t);
		float freq_t_background = lm_background.getUnigramFreq(t);

		float freq_s_foreground = lm_foreground.getUnigramFreq(s);
		float freq_s_background = lm_background.getUnigramFreq(s);

		// Calculate conditional prob p(t|s)

		float prob_t_given_s_fore = freq_st_foreground / freq_s_foreground;
		float prob_t_given_s_back = freq_st_background / freq_s_background;

		
		float prob_s_given_t_fore = freq_st_foreground / freq_t_foreground;
		// Join probabilities
		float prob_t_and_s_fore = freq_st_foreground / TOTALS.KIDS_BIGRAM;
		float prob_t_and_s_back = freq_st_background / TOTALS.ADULTS_BIGRAM;

		// uni probabilities

		float prob_t_fore = freq_t_foreground / TOTALS.KIDS_UNIGRAM;
		float prob_t_back = freq_t_background / TOTALS.ADULTS_UNIGRAM;

		float prob_s_fore = freq_s_foreground / TOTALS.KIDS_UNIGRAM;
		float prob_s_back = freq_s_background / TOTALS.ADULTS_UNIGRAM;

		// case in which

		// first foreground model

		if (s.equals(t)) {

			// we are going to threat the special case (s,s) as (s)
			prob_t_and_s_fore = 1.0f;
			prob_t_and_s_back = prob_t_back;
			prob_t_given_s_fore = 1.0f;
			 prob_s_given_t_fore=1.0f;
			// score = Float.MIN_VALUE;
			// prob_t_given_s_fore= prob_t_fore;

		}

		float score_fore = lambda * prob_t_and_s_fore + (1 - lambda)
				* prob_t_fore;
		// second background model
		float score_back = lambda * prob_t_and_s_back + (1 - lambda)
				* prob_t_back;
		// generate mix model

		// t term idf, less frequent t are more discriminative and should va
		// larger weight
		float idf = TOTALS.KIDS_TEENS_UNIGRAM / freq_t_foreground;

		idf = (float) Math.log(idf);

		// score_fore = score_fore*idf;

		// score_back = score_back*idf;

		float query_weight_t = 0.0f;
		if (query_weight) {

			query_weight_t = query_hash.get(t) / query_total;
			// System.out.println(query_weight_t + "\t" + "\t");
			query_weight_t = (float) Math.log(query_weight_t);
		}

		float score = (float) (score_fore * Math.log(score_fore / score_back));

		// not mix model

		score = conditionalProbSmoothing(prob_t_given_s_fore, prob_t_fore);
		score = score/(query_hash.get(t)/query_total);
	
		
	//	System.out.println("Query hash:" + query_hash.get(t) + "\t"+ t);
		score = score * (query_hash.get(t)+1);
		
		
		score = (float) Math.log(score);
		
		score = score + query_weight_t;
		

		// score = score/ query_weight_t;
		// score = score + query_weight_t;

		if (IDF_WEIGHT) {
			score = (idf + 1f) * score;
		
		}

		// if (score == 0.0f) {
		// System.out.println("Score is 0" + s + "\t" + t + "\t" + score_back);
		// System.exit(0);

		// }

	/*	System.out.println("s:" + s + "\tt: " + t + "  lamda:" + lambda);
		System.out.println("freq_st_fore:" + freq_st_foreground
				+ "\tfreq_st_back:" + freq_st_background);
		System.out.println("freq_t_fore:" + freq_t_foreground
				+ "\tfreq_t_back:" + freq_t_background);
		System.out.println("freq_s_fore:" + freq_s_foreground
				+ "\tfreq_s_back:" + freq_s_background);
		System.out.println("prob_t_given_s:" + prob_t_given_s_fore
				+ "\tprob_t_and_s_fore:" + prob_t_and_s_fore);
		System.out.println("query_weight_t:" + query_weight_t
				+ "\tfreq_t_in_query" + query_hash.get(t));
		System.out.println("score_fore" + score_fore + "\tscore_back:"
				+ score_back);*/
		
		
	//	System.out.println("score before log:" + conditionalProbSmoothing(prob_t_given_s_fore, prob_t_fore) + "\t"+(float) Math.log(conditionalProbSmoothing(prob_t_given_s_fore, prob_t_fore)));

	//	System.out.println("score: " + score);
		boolean debug = false;

		if ((s.equals("inspiration") || s.equals("games")) && debug) {

		}

		if (Float.isInfinite(score) || Float.isNaN(score)) {

			System.out.println("Is infinite" + "\t" + query_weight_t + "\t"
					+ score_fore + "\t" + score_back);

			System.out.println("s:" + s + "\tt: " + t + "  lamda:" + lambda);
			System.out.println("freq_st_fore:" + freq_st_foreground
					+ "\tfreq_st_back:" + freq_st_background);
			System.out.println("freq_t_fore:" + freq_t_foreground
					+ "\tfreq_t_back:" + freq_t_background);
			System.out.println("prob_t_given_s:" + prob_t_given_s_fore
					+ "\tprob_t_and_s_fore:" + prob_t_and_s_fore);
			System.out.println("query_weight_t:" + query_weight_t
					+ "\tfreq_t_in_query" + query_hash.get(t));
			System.out.println("score_fore" + score_fore + "\tscore_back:"
					+ score_back);
			System.exit(1);
		}

		return score;
	}

	public float kl_score(float prob_fore, float prob_back) {

		float prob = 0.0f;

		prob = (float) (prob_fore * Math.log(prob_fore / prob_back));

		float score = prob + 2;

		return score;
	}

	public float conditionalProbSmoothing(float cond, float prior) {

		float prob = 0.0f;

		prob = lambda * cond + (1f - lambda) * prior;

		// prob = prob / prior;

		return prob;
	}

	public void clean() {

		lm_foreground.trigrams.clear();
		lm_foreground.bigrams.clear();
		lm_foreground.unigrams.clear();

		lm_background.trigrams.clear();
		lm_background.bigrams.clear();
		lm_background.unigrams.clear();

		// seed_tags.clear();
		lm_background = null;
		lm_foreground = null;

	}

	public Hashtable<String, Float> subQuery(Hashtable<String, Float> query,
			String query_text) {

		Hashtable<String, Float> qq = new Hashtable<String, Float>();

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String key = keys.nextElement();

		}
		return qq;

	}

	public static void addSingleTermsFreq(Hashtable<String, Float> query) {

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String tag = keys.nextElement();

			String tokens[] = tag.split("_");

			for (int i = 0; i < tokens.length; i++) {

				if (query.containsKey(tokens[i])) {

					query.put(tokens[i], query.get(tokens[i]) + 1);
				}

			}

		}

	}

	public static Hashtable<String, Float> normalize(
			Hashtable<String, Float> query) {

		Hashtable<String, Float> q = new Hashtable<String, Float>();
		Float m[] = findMinAndMax(query);

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String key = keys.nextElement();

			Float value = query.get(key);

			value = (value - m[0]) / (m[1] - m[0]);

			q.put(key, value);

		}

		return q;

	}

	public static Float[] findMinAndMax(Hashtable<String, Float> query) {
		Float[] m = new Float[2];

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String key = keys.nextElement();

			if (m[0] == null) {

				m[0] = m[1] = query.get(key);

			} else {

				float value = query.get(key);

				if (value < m[0]) {

					m[0] = value;
				}

				if (value > m[1]) {

					m[1] = value;
				}
			}

		}

		return m;

	}

	public static Hashtable<String, Float> calculateKL(
			Hashtable<String, Float> query) {

		Hashtable<String, Float> kl = new Hashtable<String, Float>();

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String tag = keys.nextElement();

			if (tag.length() <= 2) {

				continue;
			}

			int freq_s_foreground = tags.getFreq("kids", tag);

			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens-mteens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("teens-mteens", tag);

			int freq_s_background = tags.getFreq("adults", tag);

			freq_s_background = freq_s_background + tags.getFreq("back", tag);
			freq_s_background = freq_s_background + freq_s_foreground;

			float prob_s_fore = freq_s_foreground / (float) TOTALS.KIDS_UNIGRAM;
			float prob_s_back = freq_s_background
					/ (float) TOTALS.ADULTS_UNIGRAM;

			if (prob_s_fore == 0.0f) {

				continue;
			}

			float pmi = -1f;
			if (prob_s_fore != 0.0f) {
				pmi = (float) Math.log(prob_s_fore / prob_s_back);
				pmi = prob_s_fore * pmi;

				// float sign= Math.signum(pmi);
				// pmi = (float) Math.log(Math.abs(pmi));
				// pmi = pmi *sign;
				// pmi = prob_s_fore;

			}

			kl.put(tag, pmi);
			 System.out.println("pmi\t"+tag+"\t"+ pmi);

		}

		//return kl;
		return normalize(kl);

	}
	
	public static Hashtable<String, Float> calculateQueryProb(
			Hashtable<String, Float> query_freq, float total_freq) {

		Hashtable<String, Float> kl = new Hashtable<String, Float>();

		Enumeration<String> keys = query_freq.keys();

		while (keys.hasMoreElements()) {

			String tag = keys.nextElement();

			if (tag.length() <= 2) {

				continue;
			}

			Float freq = query_freq.get(tag);

			int freq_s_foreground = tags.getFreq("kids", tag);

			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens-mteens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("teens-mteens", tag);

			int freq_s_background = tags.getFreq("adults", tag);

			freq_s_background = freq_s_background + tags.getFreq("back", tag);
			freq_s_background = freq_s_background + freq_s_foreground;

			float prob_s_fore = freq_s_foreground / (float) TOTALS.KIDS_UNIGRAM;
			float prob_s_back = freq_s_background / (float) TOTALS.ADULTS_UNIGRAM;

			float prob_q = (float) freq / total_freq;
			// prob_q = (float)Math.log(prob_q);
			// pmi = pmi+prob_q;

			float w = (float) Math.log(1 / prob_s_fore);
			float score = prob_q * w;
		//	score = prob_q;

			if (prob_s_fore == 0.0f) {

				continue;
			}

			kl.put(tag, score);
			
			System.out.println("prob_q " + tag + "\t" + score);
		}
	//	return kl;
		return normalize(kl);

	}

	public static Hashtable<String, Float> buildQuery(String path) {

		Hashtable<String, Float> query = new Hashtable<String, Float>();
		Hashtable<String, Float> query_freq = new Hashtable<String, Float>();

		float total_freq = 0.0f;
		FileInput in = new FileInput(path);
		String line = in.readString();

		// getting query tag and frequencies in the query

		while (line != null) {
			String t[] = line.split("\t");
			String tag = t[1];
			float freq_s_foreground = tags.getFreq("kids", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens-mteens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("teens-mteens", tag);
			if (freq_s_foreground == 0) {
				line = in.readString();
				continue;
			}

			total_freq += Float.valueOf(t[2]);
			query_freq.put(t[1], Float.valueOf(t[2]));
			line = in.readString();

		}

		Hashtable<String, Float> kl = calculateKL(query_freq);
		Hashtable<String, Float> query_prob = calculateQueryProb(query_freq,
				total_freq);

		Float THRESHOLD = 1f;

		Enumeration<String> keys = kl.keys();

		while (keys.hasMoreElements()) {

			String tag = keys.nextElement();
			float kl_value = kl.get(tag);

			float qp_value = query_prob.get(tag);
			if (tag.length() <= 2) {

				continue;
			}

			float freq = query_freq.get(tag);
			if (freq >= THRESHOLD) {

				float score = qp_value;

				score = score * (kl_value);
				score = score;
			//	System.out.println(tag+"\t"+prob_s_fore + "\t"+ prob_s_back + "\t"+ pmi + "\t" +prob_q + "\t"+ score);
				query.put(tag, score);
				// System.out.println(t[1]);
				// query.put(t[1], Float.valueOf(t[2]));

			}

		}

		Iterator<Entry<String, Float>> sorted = Sorting.sortHashNumericValues(
				query, false);

		Hashtable<String, Float> q_sorted = new Hashtable<String, Float>();

		int threshold = 20;

		int j = 0;
		while (sorted.hasNext() && j < threshold) {
			j++;
			Entry<String, Float> n = sorted.next();
			System.out.println(n.getKey() + "\t" + query_freq.get(n.getKey()));
			q_sorted.put(n.getKey(), query_freq.get(n.getKey()));

		}

		return q_sorted;
	}

	public static void main(String args[]) throws IOException, ParseException {

		LMSearcherMix trigram_searcher = new LMSearcherMix(trigram_index);
		LMSearcherMix bigram_searcher = new LMSearcherMix(bigram_index);
		String path_query = "/home/sergio/data/results/single_query/star_wars.txt";

		initTagsFrequency(index_raw);
		Hashtable<String, Float> query = new Hashtable<String, Float>();
		// query.put("dora", 1f);
		query = TagRankingMix.buildQuery(path_query);
		
		
	//	query = normalize(query);

		// query.put("doll", 5f);

		TagRankingMix ranker = new TagRankingMix(trigram_searcher,			bigram_searcher, "teens-mteens", "adults", "kids-teens-mteens",
				query, "star wars");

		 ranker.initLanguageModel(query);
		Iterator<Entry<String, Float>> scores = null;

		scores = ranker.rankSuggestions();
		int n = 50;
		int j = 0;

		while (scores.hasNext() && j < n) {

			j++;
			Entry<String, Float> entry = scores.next();
			System.out.println(j + "\t" + entry.getKey() + "\t"
					+ entry.getValue());

	}
//
	}

}
